#!/usr/bin/perl
use warnings;
use strict;
BEGIN {
    unshift @INC, '../modules';
}
use File::Path;
use File::Spec::Functions;
use Storable;
use YAML;
use TFIDF;

our %params = (
    dmoz_dir          => '/home/xuqy/dmoz',
    sample_tfidf_file => 'data/tfidf',
    idf_yml_file      => 'data/idf.yml',
    svm_learn         => 'tools/svm_learn',
    svm_classify      => 'tools/svm_classify',
    download_script   => 'scripts/batch_download.pl',

    topics_dir        => 'topics',
    list_file         => 'list',
    threads_cnt       => 50,
    classifier_file   => 'classifier',
    seeds_file        => 'seeds',
    dump_dir          => 'dump',
    tfidf_file        => 'tfidf',
    min_page_len      => 1_000,
    max_page_len      => 128_000,
);

unless ( @ARGV == 1 ) {
    print STDERR "Usage: $0 <topic>\n";
    die;
}

my $dmoz_dir = $params{dmoz_dir};
chdir $dmoz_dir or die "$!";

$params{$_} = File::Spec->rel2abs( $params{$_} )
  for qw(download_script sample_tfidf_file idf_yml_file svm_learn svm_classify);

my $topic = shift;

die "Your dir doesn't exists!\n"
  unless -d $topic;

( my $topic_dir = $topic ) =~ s/\//\./g;
my $topics_dir = $params{topics_dir};
mkdir $topics_dir unless -d $topics_dir;
$topic_dir = "$topics_dir/$topic_dir";
mkdir $topic_dir unless -d $topic_dir;

print "Collecting seed URLs...\n";
my %seed_urls = %{ &collect_seeds($topic) };
chdir $topic_dir or die "$!";
mkdir $params{dump_dir} unless -d $params{dump_dir};

open SEEDS, "> $params{seeds_file}" or die "$!";
print SEEDS $_, "\n" for keys %seed_urls;
close SEEDS;
printf "Done! %d seeds URLs collected.\n", scalar( keys %seed_urls );

print "Start downloading seed URLs...\n";
system $params{download_script}, $params{seeds_file}, $params{dump_dir}, $params{threads_cnt};
print "Done!\n";
die;

print "Start building tfidf file...\n";

open TFIDF,        "> $params{tfidf_file}"       or die "$!";

my $idf_ref = YAML::LoadFile($params{idf_yml_file});
my $total = $idf_ref->{TOTAL};

my $dump_dir = $params{dump_dir};
opendir DIR, $dump_dir or die "$!";
foreach my $entry ( readdir DIR ) {
    next if $entry =~ /^\.{1,2}$/;
    my $file = "$dump_dir/$entry";
    next if -s $file < $params{min_page_len} or -s $file > $params{max_page_len};
    open my $in, '<', $file or die "$!";
    my $html = do { local $/; <$in> };
    my $tfidf_ref = TFIDF::html2tfidf($html, $idf_ref, $total);
    if ( @$tfidf_ref ) {
        print TFIDF '+1 ';
        for ( @$tfidf_ref ) {
            printf TFIDF "%d:%.6f ", @$_;
        }
        print TFIDF "\n";
    }
    close $in;
}
closedir DIR;

open SAMPLE_TFIDF, "< $params{sample_tfidf_file}" or die "$!";
while (<SAMPLE_TFIDF>) {
    print TFIDF "-1 $_";
}
close SAMPLE_TFIDF;
close TFIDF;

print "Done!\n";

print "Start LPU learning...\n";

my @index2pair;
while ( my ( $token, $ref ) = each %$idf_ref ) {
    next if $token eq 'TOTAL';
    my ( $index, $idf ) = @$ref;
    $index2pair[$index] = [ $token, $idf ];
}
undef $idf_ref;

open IN,       "< tfidf"    or die "$!";
open TRAIN,    "> train"    or die "$!";
open VALIDATE, "> validate" or die "$!";

my $count = 0;
while (<IN>) {
    $count++;
    if ( $count % 3 == 0 ) {
        print VALIDATE;
    }
    else {
        print TRAIN;
    }
}
close IN;
close TRAIN;
close VALIDATE;

my $SVM;
my ($c, $j);
my ($svm_learn, $svm_classify) = @params{'svm_learn', 'svm_classify'};
my @history;
while (1) {
    print_history();
    print "Please input parameter c: ";
    chomp($c = <STDIN>);
    print "Please input parameter j: ";
    chomp($j = <STDIN>);
    system $svm_learn, '-c', $c, '-j', $j, 'train', 'model';
    my $validation_result = `$svm_classify validate model`;
    my ($precision, $recall) = $validation_result =~ m{([\d.]+)%/([\d.]+)%};
    my $value = $recall * $precision; 
    push @history, [$c, $j, $value];
    print_history();
    print "Validating value is $value\n";
    print "Are you satisfied with your clasifier's performance on the validation set? (y/n)\n";
    my $yesOrNo = <STDIN>;
    last if $yesOrNo =~ /^y/i;
}
print "Final parameters: c = $c, j = $j\n";

system $svm_learn, '-c', $c, '-j', $j, 'tfidf', 'model';
print "Done!\n";

my $bias;
open IN, "< model" or die "$!";
while (<IN>) {
  if (/^(\S+) # threshold/) {
    $bias = $1;
    last;
  }
}

my @weights;

while (<IN>) {
  /^(\S+) (.+) #/;
  my $alpha = $1;
  my $pairs = $2;
  next unless defined $pairs;
  while ($pairs =~ /(\S+):(\S+)/g) {
    $weights[$1] += $alpha * $2;
  }
}
close IN;
unlink 'model', 'train', 'validate', 'svm_predictions';

my %classifier;
$classifier{TOTAL} = $total;
$classifier{BIAS} = $bias;

for (1..$#weights) {
  next unless $weights[$_];
  my ($token, $idf) = @{$index2pair[$_]};
  $classifier{$token} = [$weights[$_], $idf];
}

Storable::store(\%classifier, $params{classifier_file});

print "Done!\n";

sub collect_seeds {
    my ($topic_root) = shift;
    my $seeds_ref = {};
    my @dirs = ( $topic_root );
    while (@dirs) {
        my $dir = shift @dirs;
        print $dir, "\n";
        next unless defined $dir;
        my $list_file = catfile( $dir, $params{list_file} );
        if ( -e $list_file ) {
            open IN, "< $list_file" or die "$!";
            while (<IN>) {
                chomp;
                $seeds_ref->{$_} = 1;
            }
            close IN;
        }
        opendir DIR, $dir or die "$!";
        my @sub_dirs;
        foreach my $entry ( readdir DIR ) {
            next if $entry =~ /^\.{1,2}$/;
            my $sub_dir = catfile( $dir, $entry );
            push @sub_dirs, $sub_dir if -d $sub_dir and not -l $sub_dir;
        }
        closedir DIR;
        unshift @dirs, @sub_dirs;
    }
    return $seeds_ref;
}

sub print_history {
    if (@history) {
        print '*' x 25, "\n";
        print "History:\n";
        for (0..$#history) {
            my $history = $history[$_];
            printf "No.%d c = %f j = %f value = %f\n", $_, @$history;
        }
        print '*' x 25, "\n";
    }
}
